package poem_info2.Task;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import poem_info2.JdbcUtils.JDBCUtils;
import poem_info2.JdbcUtils.ParseDetailedUrl3;
import poem_info2.domain.Poem3;

import java.net.SocketTimeoutException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

/**
 * https://www.gushiwen.com/type/n/xianqin/n/n.html
 * 古诗文网 爬取古籍
 */
public class Task3 {
    public static int count = 1;
    public static int exception_count = 0;
    public static int reserved = 0;
    public static int existed = 0;

    public static void main(String[] args) throws Exception {
        final long start = System.currentTimeMillis();
        //存储每一次的poem对象
        JdbcTemplate template = new JdbcTemplate(JDBCUtils.getDataSource());
        List<Poem3> ps = template.query("select title, dynasty_id, writer_id from poems",
                new BeanPropertyRowMapper<>(Poem3.class));
        //存储url
        List<String> type_list = new ArrayList<String>();
        //解析每个朝代页面的url
        Document parse = Jsoup.parse(new URL("https://www.gushiwen.com/type/n/xianqin/n/n.html"),
                100000);
        Elements hrefs = parse.select("div.link2 > a");
        for (Element href : hrefs) {
            String s = href.attr("href").split("/")[3];
            type_list.add(s);
        }
        int i = 0;
        for (String ty : type_list) {
            //获取每个类别诗歌的url页面，里面包含了每首诗歌的详情页url
            System.out.println("-------------------------------------------------分割线-----------------------------------------------------");
            System.out.println(ty);
            if (ty.equals("xianqin") || ty.equals("lianghan") ||
                    ty.equals("weijin") || ty.equals("nanbeichao") || ty.equals("suidai")) continue;
            boolean flag = true;
            if (ty.equals("tangdai")) i = 2200;
            while (flag) {
                String url = "https://www.gushiwen.com/type/n/" + ty + "/n/" + ++i + ".html";
                System.out.println("正在解析: " + url);
                //获取每一首诗歌的url
                Document parse1 = null;
                try {
                    parse1 = Jsoup.parse(new URL(url), 10000);
                } catch (SocketTimeoutException e) {
                    System.out.println(url + "read time out");
                    continue;
                }
                Elements lis = parse1.select("#main > ul > li");
                for (Element li : lis) {
                    ps = template.query("select title, dynasty_id, writer_id from poems",
                            new BeanPropertyRowMapper<>(Poem3.class));
                    String href = "https://www.gushiwen.com/" + li.select("div.ycd > h2 > a").attr("href");
                    new ParseDetailedUrl3().parseUrl(href, ps);
                }
                //获取最后一个a标签的text
                String text = parse1.select("div.fenye > a").last().text();
                //text为最后页说明还未到达最后一页
                if (text.equals("最后页")) flag = true;
                else {
                    flag = false;
                }
                long end = System.currentTimeMillis();
                System.out.println("已保存: " + reserved + "！");
                System.out.println("已存在: " + existed + "！");
                System.out.println("无法解析: " + exception_count + "！");
                System.out.println("已耗时: " + (end - start) / 1000 + "s!");
            }
            i = 0;
        }
    }
}
